
<!DOCTYPE html>

<html xmlns="http://www.w3.org/1999/xhtml">
  <head>
    <meta charset="utf-8" />
  
  <!-- Licensed under the Apache 2.0 License -->
  <link rel="stylesheet" type="text/css" href="../../../_static/fonts/open-sans/stylesheet.css" />
  <!-- Licensed under the SIL Open Font License -->
  <link rel="stylesheet" type="text/css" href="../../../_static/fonts/source-serif-pro/source-serif-pro.css" />
  <link rel="stylesheet" type="text/css" href="../../../_static/css/bootstrap.min.css" />
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  
    <title>jmetal.lab.experiment &#8212; jMetalPy 1.5.3 documentation</title>
    <link rel="stylesheet" href="../../../_static/guzzle.css" type="text/css" />
    <link rel="stylesheet" href="../../../_static/pygments.css" type="text/css" />
    <link rel="stylesheet" type="text/css" href="../../../_static/custom.css" />
    <script type="text/javascript" id="documentation_options" data-url_root="../../../" src="../../../_static/documentation_options.js"></script>
    <script type="text/javascript" src="../../../_static/jquery.js"></script>
    <script type="text/javascript" src="../../../_static/underscore.js"></script>
    <script type="text/javascript" src="../../../_static/doctools.js"></script>
    <script type="text/javascript" src="../../../_static/language_data.js"></script>
    <script crossorigin="anonymous" integrity="sha256-Ae2Vz/4ePdIu6ZyI/5ZGsYnb+m0JlOmKPjt6XZ9JJkA=" type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/require.js/2.3.4/require.min.js"></script>
    <script async="async" type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/latest.js?config=TeX-AMS-MML_HTMLorMML"></script>
    <script type="text/x-mathjax-config">MathJax.Hub.Config({"tex2jax": {"inlineMath": [["$", "$"], ["\\(", "\\)"]], "processEscapes": true, "ignoreClass": "document", "processClass": "math|output_area"}})</script>
    <link rel="index" title="Index" href="../../../genindex.html" />
    <link rel="search" title="Search" href="../../../search.html" />
  
   

  </head><body>
    <div class="related" role="navigation" aria-label="related navigation">
      <h3>Navigation</h3>
      <ul>
        <li class="right" style="margin-right: 10px">
          <a href="../../../genindex.html" title="General Index"
             accesskey="I">index</a></li>
        <li class="right" >
          <a href="../../../py-modindex.html" title="Python Module Index"
             >modules</a> |</li>
        <li class="nav-item nav-item-0"><a href="../../../index.html">jMetalPy 1.5.3 documentation</a> &#187;</li>
          <li class="nav-item nav-item-1"><a href="../../index.html" accesskey="U">Module code</a> &#187;</li> 
      </ul>
    </div>
    <div class="container-wrapper">

      <div id="mobile-toggle">
        <a href="#"><span class="glyphicon glyphicon-align-justify" aria-hidden="true"></span></a>
      </div>
  <div id="left-column">
    <div class="sphinxsidebar"><a href="
    ../../../index.html" class="text-logo">
    <img src="_static/jmetalpy.png" class="img-fluid" alt="jMetalPy 1.5.3 documentation">
    <br>
</a>

<div class="sidebar-block">
  <div class="sidebar-wrapper">
    Python version of the jMetal framework
  </div>
</div>
<div class="sidebar-block">
  <div class="sidebar-wrapper">
    <h2>Table Of Contents</h2>
  </div>
  <div class="sidebar-toc">
    
    
      <ul>
<li class="toctree-l1"><a class="reference internal" href="../../../tutorials.html">Getting started</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../multiobjective.algorithms.html">Multi-objective algorithms</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../singleobjective.algorithms.html">Single-objective algorithms</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../operators.html">Operators</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../problems.html">Problems</a></li>
</ul>

    
  </div>
</div>
<div class="sidebar-block">
  <div class="sidebar-wrapper">
    <div id="main-search">
      <form class="form-inline" action="../../../search.html" method="GET" role="form">
        <div class="input-group">
          <input name="q" type="text" class="form-control" placeholder="Search...">
        </div>
        <input type="hidden" name="check_keywords" value="yes" />
        <input type="hidden" name="area" value="default" />
      </form>
    </div>
  </div>
</div>
      
    </div>
  </div>
        <div id="right-column">
          
          <nav aria-label="breadcrumb">
            <ol class="breadcrumb">
              <li class="breadcrumb-item"><a href="../../../index.html">Docs</a></li>
              
              <li class="breadcrumb-item"><a href="../../index.html">Module code</a></li>
              
              <li class="breadcrumb-item">jmetal.lab.experiment</li>
            </ol>
          </nav>
          
          <div class="document clearer body">
            
  <h1>Source code for jmetal.lab.experiment</h1><div class="highlight"><pre>
<span></span><span class="kn">import</span> <span class="nn">io</span>
<span class="kn">import</span> <span class="nn">logging</span>
<span class="kn">import</span> <span class="nn">os</span>
<span class="kn">from</span> <span class="nn">concurrent.futures</span> <span class="kn">import</span> <span class="n">ProcessPoolExecutor</span>
<span class="kn">from</span> <span class="nn">pathlib</span> <span class="kn">import</span> <span class="n">Path</span>
<span class="kn">from</span> <span class="nn">statistics</span> <span class="kn">import</span> <span class="n">median</span>
<span class="kn">from</span> <span class="nn">typing</span> <span class="kn">import</span> <span class="n">List</span>

<span class="kn">import</span> <span class="nn">matplotlib.pyplot</span> <span class="k">as</span> <span class="nn">plt</span>
<span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
<span class="kn">import</span> <span class="nn">pandas</span> <span class="k">as</span> <span class="nn">pd</span>
<span class="kn">from</span> <span class="nn">scipy.stats</span> <span class="kn">import</span> <span class="n">mannwhitneyu</span>

<span class="kn">from</span> <span class="nn">jmetal.core.algorithm</span> <span class="kn">import</span> <span class="n">Algorithm</span>
<span class="kn">from</span> <span class="nn">jmetal.core.quality_indicator</span> <span class="kn">import</span> <span class="n">QualityIndicator</span>
<span class="kn">from</span> <span class="nn">jmetal.util.solution</span> <span class="kn">import</span> <span class="n">print_function_values_to_file</span><span class="p">,</span> <span class="n">print_variables_to_file</span><span class="p">,</span> <span class="n">read_solutions</span>

<span class="n">LOGGER</span> <span class="o">=</span> <span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">(</span><span class="s1">&#39;jmetal&#39;</span><span class="p">)</span>

<span class="sd">&quot;&quot;&quot;</span>
<span class="sd">.. module:: laboratory</span>
<span class="sd">   :platform: Unix, Windows</span>
<span class="sd">   :synopsis: Run experiments. WIP!</span>

<span class="sd">.. moduleauthor:: Antonio Benítez-Hidalgo &lt;antonio.b@uma.es&gt;</span>
<span class="sd">&quot;&quot;&quot;</span>


<div class="viewcode-block" id="Job"><a class="viewcode-back" href="../../../tutorials/experiment.html#jmetal.lab.experiment.Job">[docs]</a><span class="k">class</span> <span class="nc">Job</span><span class="p">:</span>

    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">algorithm</span><span class="p">:</span> <span class="n">Algorithm</span><span class="p">,</span> <span class="n">algorithm_tag</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">problem_tag</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">run</span><span class="p">:</span> <span class="nb">int</span><span class="p">):</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">algorithm</span> <span class="o">=</span> <span class="n">algorithm</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">algorithm_tag</span> <span class="o">=</span> <span class="n">algorithm_tag</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">problem_tag</span> <span class="o">=</span> <span class="n">problem_tag</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">run_tag</span> <span class="o">=</span> <span class="n">run</span>

<div class="viewcode-block" id="Job.execute"><a class="viewcode-back" href="../../../tutorials/experiment.html#jmetal.lab.experiment.Job.execute">[docs]</a>    <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">output_path</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s1">&#39;&#39;</span><span class="p">):</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">algorithm</span><span class="o">.</span><span class="n">run</span><span class="p">()</span>

        <span class="k">if</span> <span class="n">output_path</span><span class="p">:</span>
            <span class="n">file_name</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">output_path</span><span class="p">,</span> <span class="s1">&#39;FUN.</span><span class="si">{}</span><span class="s1">.tsv&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">run_tag</span><span class="p">))</span>
            <span class="n">print_function_values_to_file</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">algorithm</span><span class="o">.</span><span class="n">get_result</span><span class="p">(),</span> <span class="n">filename</span><span class="o">=</span><span class="n">file_name</span><span class="p">)</span>

            <span class="n">file_name</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">output_path</span><span class="p">,</span> <span class="s1">&#39;VAR.</span><span class="si">{}</span><span class="s1">.tsv&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">run_tag</span><span class="p">))</span>
            <span class="n">print_variables_to_file</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">algorithm</span><span class="o">.</span><span class="n">get_result</span><span class="p">(),</span> <span class="n">filename</span><span class="o">=</span><span class="n">file_name</span><span class="p">)</span>

            <span class="n">file_name</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">output_path</span><span class="p">,</span> <span class="s1">&#39;TIME.</span><span class="si">{}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">run_tag</span><span class="p">))</span>
            <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">file_name</span><span class="p">,</span> <span class="s1">&#39;w+&#39;</span><span class="p">)</span> <span class="k">as</span> <span class="n">of</span><span class="p">:</span>
                <span class="n">of</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">algorithm</span><span class="o">.</span><span class="n">total_computing_time</span><span class="p">))</span></div></div>


<div class="viewcode-block" id="Experiment"><a class="viewcode-back" href="../../../tutorials/experiment.html#jmetal.lab.experiment.Experiment">[docs]</a><span class="k">class</span> <span class="nc">Experiment</span><span class="p">:</span>

    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">output_dir</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">jobs</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">Job</span><span class="p">],</span> <span class="n">m_workers</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">6</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot; Run an experiment to execute a list of jobs.</span>

<span class="sd">        :param output_dir: Base directory where each job will save its results.</span>
<span class="sd">        :param jobs: List of Jobs (from :py:mod:`jmetal.util.laboratory)`) to be executed.</span>
<span class="sd">        :param m_workers: Maximum number of workers to execute the Jobs in parallel.</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">jobs</span> <span class="o">=</span> <span class="n">jobs</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">m_workers</span> <span class="o">=</span> <span class="n">m_workers</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">output_dir</span> <span class="o">=</span> <span class="n">output_dir</span>

<div class="viewcode-block" id="Experiment.run"><a class="viewcode-back" href="../../../tutorials/experiment.html#jmetal.lab.experiment.Experiment.run">[docs]</a>    <span class="k">def</span> <span class="nf">run</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
        <span class="k">with</span> <span class="n">ProcessPoolExecutor</span><span class="p">(</span><span class="n">max_workers</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">m_workers</span><span class="p">)</span> <span class="k">as</span> <span class="n">executor</span><span class="p">:</span>
            <span class="k">for</span> <span class="n">job</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">jobs</span><span class="p">:</span>
                <span class="n">output_path</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">output_dir</span><span class="p">,</span> <span class="n">job</span><span class="o">.</span><span class="n">algorithm_tag</span><span class="p">,</span> <span class="n">job</span><span class="o">.</span><span class="n">problem_tag</span><span class="p">)</span>
                <span class="n">executor</span><span class="o">.</span><span class="n">submit</span><span class="p">(</span><span class="n">job</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">output_path</span><span class="p">))</span></div></div>


<div class="viewcode-block" id="generate_summary_from_experiment"><a class="viewcode-back" href="../../../tutorials/experiment.html#jmetal.lab.experiment.generate_summary_from_experiment">[docs]</a><span class="k">def</span> <span class="nf">generate_summary_from_experiment</span><span class="p">(</span><span class="n">input_dir</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">quality_indicators</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">QualityIndicator</span><span class="p">],</span>
                                     <span class="n">reference_fronts</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s1">&#39;&#39;</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot; Compute a list of quality indicators. The input data directory *must* met the following structure (this is generated</span>
<span class="sd">    automatically by the Experiment class):</span>

<span class="sd">    * &lt;base_dir&gt;</span>

<span class="sd">      * algorithm_a</span>

<span class="sd">        * problem_a</span>

<span class="sd">          * FUN.0.tsv</span>
<span class="sd">          * FUN.1.tsv</span>
<span class="sd">          * VAR.0.tsv</span>
<span class="sd">          * VAR.1.tsv</span>
<span class="sd">          * ...</span>

<span class="sd">    :param input_dir: Directory where all the input data is found (function values and variables).</span>
<span class="sd">    :param reference_fronts: Directory where reference fronts are found.</span>
<span class="sd">    :param quality_indicators: List of quality indicators to compute.</span>
<span class="sd">    :return: None.</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="k">if</span> <span class="ow">not</span> <span class="n">quality_indicators</span><span class="p">:</span>
        <span class="n">quality_indicators</span> <span class="o">=</span> <span class="p">[]</span>

    <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="s1">&#39;QualityIndicatorSummary.csv&#39;</span><span class="p">,</span> <span class="s1">&#39;w+&#39;</span><span class="p">)</span> <span class="k">as</span> <span class="n">of</span><span class="p">:</span>
        <span class="n">of</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="s1">&#39;Algorithm,Problem,ExecutionId,IndicatorName,IndicatorValue</span><span class="se">\n</span><span class="s1">&#39;</span><span class="p">)</span>

    <span class="k">for</span> <span class="n">dirname</span><span class="p">,</span> <span class="n">_</span><span class="p">,</span> <span class="n">filenames</span> <span class="ow">in</span> <span class="n">os</span><span class="o">.</span><span class="n">walk</span><span class="p">(</span><span class="n">input_dir</span><span class="p">):</span>
        <span class="k">for</span> <span class="n">filename</span> <span class="ow">in</span> <span class="n">filenames</span><span class="p">:</span>
            <span class="k">try</span><span class="p">:</span>
                <span class="c1"># Linux filesystem</span>
                <span class="n">algorithm</span><span class="p">,</span> <span class="n">problem</span> <span class="o">=</span> <span class="n">dirname</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;/&#39;</span><span class="p">)[</span><span class="o">-</span><span class="mi">2</span><span class="p">:]</span>
            <span class="k">except</span> <span class="ne">ValueError</span><span class="p">:</span>
                <span class="c1"># Windows filesystem</span>
                <span class="n">algorithm</span><span class="p">,</span> <span class="n">problem</span> <span class="o">=</span> <span class="n">dirname</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;</span><span class="se">\\</span><span class="s1">&#39;</span><span class="p">)[</span><span class="o">-</span><span class="mi">2</span><span class="p">:]</span>

            <span class="k">if</span> <span class="s1">&#39;TIME&#39;</span> <span class="ow">in</span> <span class="n">filename</span><span class="p">:</span>
                <span class="n">run_tag</span> <span class="o">=</span> <span class="p">[</span><span class="n">s</span> <span class="k">for</span> <span class="n">s</span> <span class="ow">in</span> <span class="n">filename</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;.&#39;</span><span class="p">)</span> <span class="k">if</span> <span class="n">s</span><span class="o">.</span><span class="n">isdigit</span><span class="p">()]</span><span class="o">.</span><span class="n">pop</span><span class="p">()</span>

                <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">dirname</span><span class="p">,</span> <span class="n">filename</span><span class="p">),</span> <span class="s1">&#39;r&#39;</span><span class="p">)</span> <span class="k">as</span> <span class="n">content_file</span><span class="p">:</span>
                    <span class="n">content</span> <span class="o">=</span> <span class="n">content_file</span><span class="o">.</span><span class="n">read</span><span class="p">()</span>

                <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="s1">&#39;QualityIndicatorSummary.csv&#39;</span><span class="p">,</span> <span class="s1">&#39;a+&#39;</span><span class="p">)</span> <span class="k">as</span> <span class="n">of</span><span class="p">:</span>
                    <span class="n">of</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="s1">&#39;,&#39;</span><span class="o">.</span><span class="n">join</span><span class="p">([</span><span class="n">algorithm</span><span class="p">,</span> <span class="n">problem</span><span class="p">,</span> <span class="n">run_tag</span><span class="p">,</span> <span class="s1">&#39;Time&#39;</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="n">content</span><span class="p">)]))</span>
                    <span class="n">of</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="s1">&#39;</span><span class="se">\n</span><span class="s1">&#39;</span><span class="p">)</span>

            <span class="k">if</span> <span class="s1">&#39;FUN&#39;</span> <span class="ow">in</span> <span class="n">filename</span><span class="p">:</span>
                <span class="n">solutions</span> <span class="o">=</span> <span class="n">read_solutions</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">dirname</span><span class="p">,</span> <span class="n">filename</span><span class="p">))</span>
                <span class="n">run_tag</span> <span class="o">=</span> <span class="p">[</span><span class="n">s</span> <span class="k">for</span> <span class="n">s</span> <span class="ow">in</span> <span class="n">filename</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;.&#39;</span><span class="p">)</span> <span class="k">if</span> <span class="n">s</span><span class="o">.</span><span class="n">isdigit</span><span class="p">()]</span><span class="o">.</span><span class="n">pop</span><span class="p">()</span>

                <span class="k">for</span> <span class="n">indicator</span> <span class="ow">in</span> <span class="n">quality_indicators</span><span class="p">:</span>
                    <span class="n">reference_front_file</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">reference_fronts</span><span class="p">,</span> <span class="n">problem</span> <span class="o">+</span> <span class="s1">&#39;.pf&#39;</span><span class="p">)</span>

                    <span class="c1"># Add reference front if any</span>
                    <span class="k">if</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">indicator</span><span class="p">,</span> <span class="s1">&#39;reference_front&#39;</span><span class="p">):</span>
                        <span class="k">if</span> <span class="n">Path</span><span class="p">(</span><span class="n">reference_front_file</span><span class="p">)</span><span class="o">.</span><span class="n">is_file</span><span class="p">():</span>
                            <span class="n">indicator</span><span class="o">.</span><span class="n">reference_front</span> <span class="o">=</span> <span class="n">read_solutions</span><span class="p">(</span><span class="n">reference_front_file</span><span class="p">)</span>
                        <span class="k">else</span><span class="p">:</span>
                            <span class="n">LOGGER</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s1">&#39;Reference front not found at&#39;</span><span class="p">,</span> <span class="n">reference_front_file</span><span class="p">)</span>

                    <span class="n">result</span> <span class="o">=</span> <span class="n">indicator</span><span class="o">.</span><span class="n">compute</span><span class="p">(</span><span class="n">solutions</span><span class="p">)</span>

                    <span class="c1"># Save quality indicator value to file</span>
                    <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="s1">&#39;QualityIndicatorSummary.csv&#39;</span><span class="p">,</span> <span class="s1">&#39;a+&#39;</span><span class="p">)</span> <span class="k">as</span> <span class="n">of</span><span class="p">:</span>
                        <span class="n">of</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="s1">&#39;,&#39;</span><span class="o">.</span><span class="n">join</span><span class="p">([</span><span class="n">algorithm</span><span class="p">,</span> <span class="n">problem</span><span class="p">,</span> <span class="n">run_tag</span><span class="p">,</span> <span class="n">indicator</span><span class="o">.</span><span class="n">get_name</span><span class="p">(),</span> <span class="nb">str</span><span class="p">(</span><span class="n">result</span><span class="p">)]))</span>
                        <span class="n">of</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="s1">&#39;</span><span class="se">\n</span><span class="s1">&#39;</span><span class="p">)</span></div>


<span class="k">def</span> <span class="nf">generate_boxplot</span><span class="p">(</span><span class="n">filename</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">output_dir</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s1">&#39;boxplot&#39;</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot; Generate boxplot diagrams.</span>

<span class="sd">    :param filename: Input filename (summary).</span>
<span class="sd">    :param output_dir: Output path.</span>
<span class="sd">    &quot;&quot;&quot;</span>
    <span class="n">df</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">read_csv</span><span class="p">(</span><span class="n">filename</span><span class="p">,</span> <span class="n">skipinitialspace</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>

    <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="nb">set</span><span class="p">(</span><span class="n">df</span><span class="o">.</span><span class="n">columns</span><span class="o">.</span><span class="n">tolist</span><span class="p">()))</span> <span class="o">!=</span> <span class="mi">5</span><span class="p">:</span>
        <span class="k">raise</span> <span class="ne">Exception</span><span class="p">(</span><span class="s1">&#39;Wrong number of columns&#39;</span><span class="p">)</span>

    <span class="k">if</span> <span class="n">Path</span><span class="p">(</span><span class="n">output_dir</span><span class="p">)</span><span class="o">.</span><span class="n">is_dir</span><span class="p">():</span>
        <span class="n">LOGGER</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s1">&#39;Directory </span><span class="si">{}</span><span class="s1"> exists. Removing contents.&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">output_dir</span><span class="p">))</span>
        <span class="k">for</span> <span class="n">file</span> <span class="ow">in</span> <span class="n">os</span><span class="o">.</span><span class="n">listdir</span><span class="p">(</span><span class="n">output_dir</span><span class="p">):</span>
            <span class="n">os</span><span class="o">.</span><span class="n">remove</span><span class="p">(</span><span class="s1">&#39;</span><span class="si">{0}</span><span class="s1">/</span><span class="si">{1}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">output_dir</span><span class="p">,</span> <span class="n">file</span><span class="p">))</span>
    <span class="k">else</span><span class="p">:</span>
        <span class="n">LOGGER</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s1">&#39;Directory </span><span class="si">{}</span><span class="s1"> does not exist. Creating it.&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">output_dir</span><span class="p">))</span>
        <span class="n">Path</span><span class="p">(</span><span class="n">output_dir</span><span class="p">)</span><span class="o">.</span><span class="n">mkdir</span><span class="p">(</span><span class="n">parents</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>

    <span class="n">algorithms</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">unique</span><span class="p">(</span><span class="n">df</span><span class="p">[</span><span class="s1">&#39;Algorithm&#39;</span><span class="p">])</span>
    <span class="n">problems</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">unique</span><span class="p">(</span><span class="n">df</span><span class="p">[</span><span class="s1">&#39;Problem&#39;</span><span class="p">])</span>
    <span class="n">indicators</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">unique</span><span class="p">(</span><span class="n">df</span><span class="p">[</span><span class="s1">&#39;IndicatorName&#39;</span><span class="p">])</span>

    <span class="c1"># We consider the quality indicator indicator_name</span>

    <span class="k">for</span> <span class="n">indicator_name</span> <span class="ow">in</span> <span class="n">indicators</span><span class="p">:</span>
        <span class="n">data</span> <span class="o">=</span> <span class="n">df</span><span class="p">[</span><span class="n">df</span><span class="p">[</span><span class="s1">&#39;IndicatorName&#39;</span><span class="p">]</span> <span class="o">==</span> <span class="n">indicator_name</span><span class="p">]</span>

        <span class="k">for</span> <span class="n">pr</span> <span class="ow">in</span> <span class="n">problems</span><span class="p">:</span>
            <span class="n">data_to_plot</span> <span class="o">=</span> <span class="p">[]</span>

            <span class="k">for</span> <span class="n">alg</span> <span class="ow">in</span> <span class="n">algorithms</span><span class="p">:</span>
                <span class="n">data_to_plot</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">data</span><span class="p">[</span><span class="s1">&#39;IndicatorValue&#39;</span><span class="p">][</span><span class="n">np</span><span class="o">.</span><span class="n">logical_and</span><span class="p">(</span>
                    <span class="n">data</span><span class="p">[</span><span class="s1">&#39;Algorithm&#39;</span><span class="p">]</span> <span class="o">==</span> <span class="n">alg</span><span class="p">,</span> <span class="n">data</span><span class="p">[</span><span class="s1">&#39;Problem&#39;</span><span class="p">]</span> <span class="o">==</span> <span class="n">pr</span><span class="p">)])</span>

            <span class="c1"># Create a figure instance</span>
            <span class="n">fig</span> <span class="o">=</span> <span class="n">plt</span><span class="o">.</span><span class="n">figure</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="n">figsize</span><span class="o">=</span><span class="p">(</span><span class="mi">9</span><span class="p">,</span> <span class="mi">6</span><span class="p">))</span>
            <span class="n">plt</span><span class="o">.</span><span class="n">suptitle</span><span class="p">(</span><span class="n">pr</span><span class="p">,</span> <span class="n">y</span><span class="o">=</span><span class="mf">0.95</span><span class="p">,</span> <span class="n">fontsize</span><span class="o">=</span><span class="mi">18</span><span class="p">)</span>

            <span class="n">ax</span> <span class="o">=</span> <span class="n">fig</span><span class="o">.</span><span class="n">add_subplot</span><span class="p">(</span><span class="mi">111</span><span class="p">)</span>
            <span class="n">ax</span><span class="o">.</span><span class="n">boxplot</span><span class="p">(</span><span class="n">data_to_plot</span><span class="p">)</span>

            <span class="n">ax</span><span class="o">.</span><span class="n">set_xticklabels</span><span class="p">(</span><span class="n">algorithms</span><span class="p">)</span>
            <span class="n">ax</span><span class="o">.</span><span class="n">tick_params</span><span class="p">(</span><span class="n">labelsize</span><span class="o">=</span><span class="mi">20</span><span class="p">)</span>

            <span class="n">plt</span><span class="o">.</span><span class="n">savefig</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">output_dir</span><span class="p">,</span> <span class="s1">&#39;boxplot-</span><span class="si">{}</span><span class="s1">-</span><span class="si">{}</span><span class="s1">.png&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">pr</span><span class="p">,</span> <span class="n">indicator_name</span><span class="p">)),</span> <span class="n">bbox_inches</span><span class="o">=</span><span class="s1">&#39;tight&#39;</span><span class="p">)</span>
            <span class="n">plt</span><span class="o">.</span><span class="n">savefig</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">output_dir</span><span class="p">,</span> <span class="s1">&#39;boxplot-</span><span class="si">{}</span><span class="s1">-</span><span class="si">{}</span><span class="s1">.eps&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">pr</span><span class="p">,</span> <span class="n">indicator_name</span><span class="p">)),</span> <span class="n">bbox_inches</span><span class="o">=</span><span class="s1">&#39;tight&#39;</span><span class="p">)</span>
            <span class="n">plt</span><span class="o">.</span><span class="n">close</span><span class="p">(</span><span class="n">fig</span><span class="p">)</span>


<span class="k">def</span> <span class="nf">generate_latex_tables</span><span class="p">(</span><span class="n">filename</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">output_dir</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s1">&#39;latex/statistical&#39;</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot; Computes a number of statistical values (mean, median, standard deviation, interquartile range).</span>

<span class="sd">    :param filename: Input filename (summary).</span>
<span class="sd">    :param output_dir: Output path.</span>
<span class="sd">    &quot;&quot;&quot;</span>
    <span class="n">df</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">read_csv</span><span class="p">(</span><span class="n">filename</span><span class="p">,</span> <span class="n">skipinitialspace</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>

    <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="nb">set</span><span class="p">(</span><span class="n">df</span><span class="o">.</span><span class="n">columns</span><span class="o">.</span><span class="n">tolist</span><span class="p">()))</span> <span class="o">!=</span> <span class="mi">5</span><span class="p">:</span>
        <span class="k">raise</span> <span class="ne">Exception</span><span class="p">(</span><span class="s1">&#39;Wrong number of columns&#39;</span><span class="p">)</span>

    <span class="k">if</span> <span class="n">Path</span><span class="p">(</span><span class="n">output_dir</span><span class="p">)</span><span class="o">.</span><span class="n">is_dir</span><span class="p">():</span>
        <span class="n">LOGGER</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s1">&#39;Directory </span><span class="si">{}</span><span class="s1"> exists. Removing contents.&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">output_dir</span><span class="p">))</span>
        <span class="k">for</span> <span class="n">file</span> <span class="ow">in</span> <span class="n">os</span><span class="o">.</span><span class="n">listdir</span><span class="p">(</span><span class="n">output_dir</span><span class="p">):</span>
            <span class="n">os</span><span class="o">.</span><span class="n">remove</span><span class="p">(</span><span class="s1">&#39;</span><span class="si">{0}</span><span class="s1">/</span><span class="si">{1}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">output_dir</span><span class="p">,</span> <span class="n">file</span><span class="p">))</span>
    <span class="k">else</span><span class="p">:</span>
        <span class="n">LOGGER</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s1">&#39;Directory </span><span class="si">{}</span><span class="s1"> does not exist. Creating it.&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">output_dir</span><span class="p">))</span>
        <span class="n">Path</span><span class="p">(</span><span class="n">output_dir</span><span class="p">)</span><span class="o">.</span><span class="n">mkdir</span><span class="p">(</span><span class="n">parents</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>

    <span class="c1"># Generate median &amp; iqr tables</span>
    <span class="n">median</span><span class="p">,</span> <span class="n">iqr</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">(),</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">()</span>
    <span class="n">mean</span><span class="p">,</span> <span class="n">std</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">(),</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">()</span>

    <span class="k">for</span> <span class="n">algorithm_name</span><span class="p">,</span> <span class="n">subset</span> <span class="ow">in</span> <span class="n">df</span><span class="o">.</span><span class="n">groupby</span><span class="p">(</span><span class="s1">&#39;Algorithm&#39;</span><span class="p">,</span> <span class="n">sort</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
        <span class="n">subset</span> <span class="o">=</span> <span class="n">subset</span><span class="o">.</span><span class="n">drop</span><span class="p">(</span><span class="s1">&#39;Algorithm&#39;</span><span class="p">,</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
        <span class="n">subset</span> <span class="o">=</span> <span class="n">subset</span><span class="o">.</span><span class="n">rename</span><span class="p">(</span><span class="n">columns</span><span class="o">=</span><span class="p">{</span><span class="s1">&#39;IndicatorValue&#39;</span><span class="p">:</span> <span class="n">algorithm_name</span><span class="p">})</span>
        <span class="n">subset</span> <span class="o">=</span> <span class="n">subset</span><span class="o">.</span><span class="n">set_index</span><span class="p">([</span><span class="s1">&#39;Problem&#39;</span><span class="p">,</span> <span class="s1">&#39;IndicatorName&#39;</span><span class="p">,</span> <span class="s1">&#39;ExecutionId&#39;</span><span class="p">])</span>

        <span class="c1"># Compute Median and Interquartile range</span>
        <span class="n">median_</span> <span class="o">=</span> <span class="n">subset</span><span class="o">.</span><span class="n">groupby</span><span class="p">(</span><span class="n">level</span><span class="o">=</span><span class="p">[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">])</span><span class="o">.</span><span class="n">median</span><span class="p">()</span>
        <span class="n">median</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">concat</span><span class="p">([</span><span class="n">median</span><span class="p">,</span> <span class="n">median_</span><span class="p">],</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>

        <span class="n">iqr_</span> <span class="o">=</span> <span class="n">subset</span><span class="o">.</span><span class="n">groupby</span><span class="p">(</span><span class="n">level</span><span class="o">=</span><span class="p">[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">])</span><span class="o">.</span><span class="n">quantile</span><span class="p">(</span><span class="mf">0.75</span><span class="p">)</span> <span class="o">-</span> <span class="n">subset</span><span class="o">.</span><span class="n">groupby</span><span class="p">(</span><span class="n">level</span><span class="o">=</span><span class="p">[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">])</span><span class="o">.</span><span class="n">quantile</span><span class="p">(</span><span class="mf">0.25</span><span class="p">)</span>
        <span class="n">iqr</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">concat</span><span class="p">([</span><span class="n">iqr</span><span class="p">,</span> <span class="n">iqr_</span><span class="p">],</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>

        <span class="c1"># Compute Mean and Standard deviation</span>
        <span class="n">mean_</span> <span class="o">=</span> <span class="n">subset</span><span class="o">.</span><span class="n">groupby</span><span class="p">(</span><span class="n">level</span><span class="o">=</span><span class="p">[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">])</span><span class="o">.</span><span class="n">mean</span><span class="p">()</span>
        <span class="n">mean</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">concat</span><span class="p">([</span><span class="n">mean</span><span class="p">,</span> <span class="n">mean_</span><span class="p">],</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>

        <span class="n">std_</span> <span class="o">=</span> <span class="n">subset</span><span class="o">.</span><span class="n">groupby</span><span class="p">(</span><span class="n">level</span><span class="o">=</span><span class="p">[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">])</span><span class="o">.</span><span class="n">std</span><span class="p">()</span>
        <span class="n">std</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">concat</span><span class="p">([</span><span class="n">std</span><span class="p">,</span> <span class="n">std_</span><span class="p">],</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>

    <span class="c1"># Generate mean &amp; std tables</span>
    <span class="k">for</span> <span class="n">indicator_name</span><span class="p">,</span> <span class="n">subset</span> <span class="ow">in</span> <span class="n">std</span><span class="o">.</span><span class="n">groupby</span><span class="p">(</span><span class="s1">&#39;IndicatorName&#39;</span><span class="p">,</span> <span class="n">sort</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
        <span class="n">subset</span> <span class="o">=</span> <span class="n">median</span><span class="o">.</span><span class="n">groupby</span><span class="p">(</span><span class="s1">&#39;IndicatorName&#39;</span><span class="p">,</span> <span class="n">sort</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span><span class="o">.</span><span class="n">get_group</span><span class="p">(</span><span class="n">indicator_name</span><span class="p">)</span>
        <span class="n">subset</span><span class="o">.</span><span class="n">index</span> <span class="o">=</span> <span class="n">subset</span><span class="o">.</span><span class="n">index</span><span class="o">.</span><span class="n">droplevel</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
        <span class="n">subset</span><span class="o">.</span><span class="n">to_csv</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">output_dir</span><span class="p">,</span> <span class="s1">&#39;Median-</span><span class="si">{}</span><span class="s1">.csv&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">indicator_name</span><span class="p">)),</span> <span class="n">sep</span><span class="o">=</span><span class="s1">&#39;</span><span class="se">\t</span><span class="s1">&#39;</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="s1">&#39;utf-8&#39;</span><span class="p">)</span>

        <span class="n">subset</span> <span class="o">=</span> <span class="n">iqr</span><span class="o">.</span><span class="n">groupby</span><span class="p">(</span><span class="s1">&#39;IndicatorName&#39;</span><span class="p">,</span> <span class="n">sort</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span><span class="o">.</span><span class="n">get_group</span><span class="p">(</span><span class="n">indicator_name</span><span class="p">)</span>
        <span class="n">subset</span><span class="o">.</span><span class="n">index</span> <span class="o">=</span> <span class="n">subset</span><span class="o">.</span><span class="n">index</span><span class="o">.</span><span class="n">droplevel</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
        <span class="n">subset</span><span class="o">.</span><span class="n">to_csv</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">output_dir</span><span class="p">,</span> <span class="s1">&#39;IQR-</span><span class="si">{}</span><span class="s1">.csv&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">indicator_name</span><span class="p">)),</span> <span class="n">sep</span><span class="o">=</span><span class="s1">&#39;</span><span class="se">\t</span><span class="s1">&#39;</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="s1">&#39;utf-8&#39;</span><span class="p">)</span>

        <span class="n">subset</span> <span class="o">=</span> <span class="n">mean</span><span class="o">.</span><span class="n">groupby</span><span class="p">(</span><span class="s1">&#39;IndicatorName&#39;</span><span class="p">,</span> <span class="n">sort</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span><span class="o">.</span><span class="n">get_group</span><span class="p">(</span><span class="n">indicator_name</span><span class="p">)</span>
        <span class="n">subset</span><span class="o">.</span><span class="n">index</span> <span class="o">=</span> <span class="n">subset</span><span class="o">.</span><span class="n">index</span><span class="o">.</span><span class="n">droplevel</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
        <span class="n">subset</span><span class="o">.</span><span class="n">to_csv</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">output_dir</span><span class="p">,</span> <span class="s1">&#39;Mean-</span><span class="si">{}</span><span class="s1">.csv&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">indicator_name</span><span class="p">)),</span> <span class="n">sep</span><span class="o">=</span><span class="s1">&#39;</span><span class="se">\t</span><span class="s1">&#39;</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="s1">&#39;utf-8&#39;</span><span class="p">)</span>

        <span class="n">subset</span> <span class="o">=</span> <span class="n">std</span><span class="o">.</span><span class="n">groupby</span><span class="p">(</span><span class="s1">&#39;IndicatorName&#39;</span><span class="p">,</span> <span class="n">sort</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span><span class="o">.</span><span class="n">get_group</span><span class="p">(</span><span class="n">indicator_name</span><span class="p">)</span>
        <span class="n">subset</span><span class="o">.</span><span class="n">index</span> <span class="o">=</span> <span class="n">subset</span><span class="o">.</span><span class="n">index</span><span class="o">.</span><span class="n">droplevel</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
        <span class="n">subset</span><span class="o">.</span><span class="n">to_csv</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">output_dir</span><span class="p">,</span> <span class="s1">&#39;Std-</span><span class="si">{}</span><span class="s1">.csv&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">indicator_name</span><span class="p">)),</span> <span class="n">sep</span><span class="o">=</span><span class="s1">&#39;</span><span class="se">\t</span><span class="s1">&#39;</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="s1">&#39;utf-8&#39;</span><span class="p">)</span>

    <span class="c1"># Generate LaTeX tables</span>
    <span class="k">for</span> <span class="n">indicator_name</span> <span class="ow">in</span> <span class="n">df</span><span class="o">.</span><span class="n">groupby</span><span class="p">(</span><span class="s1">&#39;IndicatorName&#39;</span><span class="p">,</span> <span class="n">sort</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span><span class="o">.</span><span class="n">groups</span><span class="o">.</span><span class="n">keys</span><span class="p">():</span>
        <span class="c1"># Median &amp; IQR</span>
        <span class="n">md</span> <span class="o">=</span> <span class="n">median</span><span class="o">.</span><span class="n">groupby</span><span class="p">(</span><span class="s1">&#39;IndicatorName&#39;</span><span class="p">,</span> <span class="n">sort</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span><span class="o">.</span><span class="n">get_group</span><span class="p">(</span><span class="n">indicator_name</span><span class="p">)</span>
        <span class="n">md</span><span class="o">.</span><span class="n">index</span> <span class="o">=</span> <span class="n">md</span><span class="o">.</span><span class="n">index</span><span class="o">.</span><span class="n">droplevel</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>

        <span class="n">i</span> <span class="o">=</span> <span class="n">iqr</span><span class="o">.</span><span class="n">groupby</span><span class="p">(</span><span class="s1">&#39;IndicatorName&#39;</span><span class="p">,</span> <span class="n">sort</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span><span class="o">.</span><span class="n">get_group</span><span class="p">(</span><span class="n">indicator_name</span><span class="p">)</span>
        <span class="n">i</span><span class="o">.</span><span class="n">index</span> <span class="o">=</span> <span class="n">i</span><span class="o">.</span><span class="n">index</span><span class="o">.</span><span class="n">droplevel</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>

        <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">output_dir</span><span class="p">,</span> <span class="s1">&#39;MedianIQR-</span><span class="si">{}</span><span class="s1">.tex&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">indicator_name</span><span class="p">)),</span> <span class="s1">&#39;w&#39;</span><span class="p">)</span> <span class="k">as</span> <span class="n">latex</span><span class="p">:</span>
            <span class="n">latex</span><span class="o">.</span><span class="n">write</span><span class="p">(</span>
                <span class="n">__averages_to_latex</span><span class="p">(</span>
                    <span class="n">md</span><span class="p">,</span>
                    <span class="n">i</span><span class="p">,</span>
                    <span class="n">caption</span><span class="o">=</span><span class="s1">&#39;Median and Interquartile Range of the </span><span class="si">{}</span><span class="s1"> quality indicator.&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">indicator_name</span><span class="p">),</span>
                    <span class="n">minimization</span><span class="o">=</span><span class="n">check_minimization</span><span class="p">(</span><span class="n">indicator_name</span><span class="p">),</span>
                    <span class="n">label</span><span class="o">=</span><span class="s1">&#39;table:</span><span class="si">{}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">indicator_name</span><span class="p">)</span>
                <span class="p">)</span>
            <span class="p">)</span>

        <span class="c1"># Mean &amp; Std</span>
        <span class="n">mn</span> <span class="o">=</span> <span class="n">mean</span><span class="o">.</span><span class="n">groupby</span><span class="p">(</span><span class="s1">&#39;IndicatorName&#39;</span><span class="p">,</span> <span class="n">sort</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span><span class="o">.</span><span class="n">get_group</span><span class="p">(</span><span class="n">indicator_name</span><span class="p">)</span>
        <span class="n">mn</span><span class="o">.</span><span class="n">index</span> <span class="o">=</span> <span class="n">mn</span><span class="o">.</span><span class="n">index</span><span class="o">.</span><span class="n">droplevel</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>

        <span class="n">s</span> <span class="o">=</span> <span class="n">std</span><span class="o">.</span><span class="n">groupby</span><span class="p">(</span><span class="s1">&#39;IndicatorName&#39;</span><span class="p">,</span> <span class="n">sort</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span><span class="o">.</span><span class="n">get_group</span><span class="p">(</span><span class="n">indicator_name</span><span class="p">)</span>
        <span class="n">s</span><span class="o">.</span><span class="n">index</span> <span class="o">=</span> <span class="n">s</span><span class="o">.</span><span class="n">index</span><span class="o">.</span><span class="n">droplevel</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>

        <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">output_dir</span><span class="p">,</span> <span class="s1">&#39;MeanStd-</span><span class="si">{}</span><span class="s1">.tex&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">indicator_name</span><span class="p">)),</span> <span class="s1">&#39;w&#39;</span><span class="p">)</span> <span class="k">as</span> <span class="n">latex</span><span class="p">:</span>
            <span class="n">latex</span><span class="o">.</span><span class="n">write</span><span class="p">(</span>
                <span class="n">__averages_to_latex</span><span class="p">(</span>
                    <span class="n">mn</span><span class="p">,</span>
                    <span class="n">s</span><span class="p">,</span>
                    <span class="n">caption</span><span class="o">=</span><span class="s1">&#39;Mean and Standard Deviation of the </span><span class="si">{}</span><span class="s1"> quality indicator.&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">indicator_name</span><span class="p">),</span>
                    <span class="n">minimization</span><span class="o">=</span><span class="n">check_minimization</span><span class="p">(</span><span class="n">indicator_name</span><span class="p">),</span>
                    <span class="n">label</span><span class="o">=</span><span class="s1">&#39;table:</span><span class="si">{}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">indicator_name</span><span class="p">)</span>
                <span class="p">)</span>
            <span class="p">)</span>


<span class="k">def</span> <span class="nf">compute_wilcoxon</span><span class="p">(</span><span class="n">filename</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">output_dir</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s1">&#39;latex/wilcoxon&#39;</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    :param filename: Input filename (summary).</span>
<span class="sd">    :param output_dir: Output path.</span>
<span class="sd">    &quot;&quot;&quot;</span>
    <span class="n">df</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">read_csv</span><span class="p">(</span><span class="n">filename</span><span class="p">,</span> <span class="n">skipinitialspace</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>

    <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="nb">set</span><span class="p">(</span><span class="n">df</span><span class="o">.</span><span class="n">columns</span><span class="o">.</span><span class="n">tolist</span><span class="p">()))</span> <span class="o">!=</span> <span class="mi">5</span><span class="p">:</span>
        <span class="k">raise</span> <span class="ne">Exception</span><span class="p">(</span><span class="s1">&#39;Wrong number of columns&#39;</span><span class="p">)</span>

    <span class="k">if</span> <span class="n">Path</span><span class="p">(</span><span class="n">output_dir</span><span class="p">)</span><span class="o">.</span><span class="n">is_dir</span><span class="p">():</span>
        <span class="n">LOGGER</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s1">&#39;Directory </span><span class="si">{}</span><span class="s1"> exists. Removing contents.&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">output_dir</span><span class="p">))</span>
        <span class="k">for</span> <span class="n">file</span> <span class="ow">in</span> <span class="n">os</span><span class="o">.</span><span class="n">listdir</span><span class="p">(</span><span class="n">output_dir</span><span class="p">):</span>
            <span class="n">os</span><span class="o">.</span><span class="n">remove</span><span class="p">(</span><span class="s1">&#39;</span><span class="si">{0}</span><span class="s1">/</span><span class="si">{1}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">output_dir</span><span class="p">,</span> <span class="n">file</span><span class="p">))</span>
    <span class="k">else</span><span class="p">:</span>
        <span class="n">LOGGER</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s1">&#39;Directory </span><span class="si">{}</span><span class="s1"> does not exist. Creating it.&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">output_dir</span><span class="p">))</span>
        <span class="n">Path</span><span class="p">(</span><span class="n">output_dir</span><span class="p">)</span><span class="o">.</span><span class="n">mkdir</span><span class="p">(</span><span class="n">parents</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>

    <span class="n">algorithms</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">unique</span><span class="p">(</span><span class="n">df</span><span class="p">[</span><span class="s1">&#39;Algorithm&#39;</span><span class="p">])</span>
    <span class="n">problems</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">unique</span><span class="p">(</span><span class="n">df</span><span class="p">[</span><span class="s1">&#39;Problem&#39;</span><span class="p">])</span>
    <span class="n">indicators</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">unique</span><span class="p">(</span><span class="n">df</span><span class="p">[</span><span class="s1">&#39;IndicatorName&#39;</span><span class="p">])</span>

    <span class="n">table</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">(</span><span class="n">index</span><span class="o">=</span><span class="n">algorithms</span><span class="p">[</span><span class="mi">0</span><span class="p">:</span><span class="o">-</span><span class="mi">1</span><span class="p">],</span> <span class="n">columns</span><span class="o">=</span><span class="n">algorithms</span><span class="p">[</span><span class="mi">1</span><span class="p">:])</span>

    <span class="k">for</span> <span class="n">indicator_name</span> <span class="ow">in</span> <span class="n">indicators</span><span class="p">:</span>
        <span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="n">row_algorithm</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">algorithms</span><span class="p">[</span><span class="mi">0</span><span class="p">:</span><span class="o">-</span><span class="mi">1</span><span class="p">]):</span>
            <span class="n">wilcoxon</span> <span class="o">=</span> <span class="p">[]</span>
            <span class="k">for</span> <span class="n">j</span><span class="p">,</span> <span class="n">col_algorithm</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">algorithms</span><span class="p">[</span><span class="mi">1</span><span class="p">:]):</span>
                <span class="n">line</span> <span class="o">=</span> <span class="p">[]</span>

                <span class="k">if</span> <span class="n">i</span> <span class="o">&lt;=</span> <span class="n">j</span><span class="p">:</span>
                    <span class="k">for</span> <span class="n">problem</span> <span class="ow">in</span> <span class="n">problems</span><span class="p">:</span>
                        <span class="n">df1</span> <span class="o">=</span> <span class="n">df</span><span class="p">[(</span><span class="n">df</span><span class="p">[</span><span class="s2">&quot;Algorithm&quot;</span><span class="p">]</span> <span class="o">==</span> <span class="n">row_algorithm</span><span class="p">)</span> <span class="o">&amp;</span> <span class="p">(</span><span class="n">df</span><span class="p">[</span><span class="s2">&quot;Problem&quot;</span><span class="p">]</span> <span class="o">==</span> <span class="n">problem</span><span class="p">)</span> <span class="o">&amp;</span> <span class="p">(</span>
                                <span class="n">df</span><span class="p">[</span><span class="s2">&quot;IndicatorName&quot;</span><span class="p">]</span> <span class="o">==</span> <span class="n">indicator_name</span><span class="p">)]</span>
                        <span class="n">df2</span> <span class="o">=</span> <span class="n">df</span><span class="p">[(</span><span class="n">df</span><span class="p">[</span><span class="s2">&quot;Algorithm&quot;</span><span class="p">]</span> <span class="o">==</span> <span class="n">col_algorithm</span><span class="p">)</span> <span class="o">&amp;</span> <span class="p">(</span><span class="n">df</span><span class="p">[</span><span class="s2">&quot;Problem&quot;</span><span class="p">]</span> <span class="o">==</span> <span class="n">problem</span><span class="p">)</span> <span class="o">&amp;</span> <span class="p">(</span>
                                <span class="n">df</span><span class="p">[</span><span class="s2">&quot;IndicatorName&quot;</span><span class="p">]</span> <span class="o">==</span> <span class="n">indicator_name</span><span class="p">)]</span>

                        <span class="n">data1</span> <span class="o">=</span> <span class="n">df1</span><span class="p">[</span><span class="s2">&quot;IndicatorValue&quot;</span><span class="p">]</span>
                        <span class="n">data2</span> <span class="o">=</span> <span class="n">df2</span><span class="p">[</span><span class="s2">&quot;IndicatorValue&quot;</span><span class="p">]</span>

                        <span class="n">median1</span> <span class="o">=</span> <span class="n">median</span><span class="p">(</span><span class="n">data1</span><span class="p">)</span>
                        <span class="n">median2</span> <span class="o">=</span> <span class="n">median</span><span class="p">(</span><span class="n">data2</span><span class="p">)</span>

                        <span class="n">stat</span><span class="p">,</span> <span class="n">p</span> <span class="o">=</span> <span class="n">mannwhitneyu</span><span class="p">(</span><span class="n">data1</span><span class="p">,</span> <span class="n">data2</span><span class="p">)</span>

                        <span class="k">if</span> <span class="n">p</span> <span class="o">&lt;=</span> <span class="mf">0.05</span><span class="p">:</span>
                            <span class="k">if</span> <span class="n">check_minimization</span><span class="p">(</span><span class="n">indicator_name</span><span class="p">):</span>
                                <span class="k">if</span> <span class="n">median1</span> <span class="o">&lt;=</span> <span class="n">median2</span><span class="p">:</span>
                                    <span class="n">line</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s1">&#39;+&#39;</span><span class="p">)</span>
                                <span class="k">else</span><span class="p">:</span>
                                    <span class="n">line</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s1">&#39;o&#39;</span><span class="p">)</span>
                            <span class="k">else</span><span class="p">:</span>
                                <span class="k">if</span> <span class="n">median1</span> <span class="o">&gt;=</span> <span class="n">median2</span><span class="p">:</span>
                                    <span class="n">line</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s1">&#39;+&#39;</span><span class="p">)</span>
                                <span class="k">else</span><span class="p">:</span>
                                    <span class="n">line</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s1">&#39;o&#39;</span><span class="p">)</span>
                        <span class="k">else</span><span class="p">:</span>
                            <span class="n">line</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s1">&#39;-&#39;</span><span class="p">)</span>
                    <span class="n">wilcoxon</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s1">&#39;&#39;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">line</span><span class="p">))</span>

            <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">wilcoxon</span><span class="p">)</span> <span class="o">&lt;</span> <span class="nb">len</span><span class="p">(</span><span class="n">algorithms</span><span class="p">):</span> <span class="n">wilcoxon</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;&#39;</span><span class="p">]</span> <span class="o">*</span> <span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">algorithms</span><span class="p">)</span> <span class="o">-</span> <span class="nb">len</span><span class="p">(</span><span class="n">wilcoxon</span><span class="p">)</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span> <span class="o">+</span> <span class="n">wilcoxon</span>
            <span class="n">table</span><span class="o">.</span><span class="n">loc</span><span class="p">[</span><span class="n">row_algorithm</span><span class="p">]</span> <span class="o">=</span> <span class="n">wilcoxon</span>

        <span class="n">table</span><span class="o">.</span><span class="n">to_csv</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">output_dir</span><span class="p">,</span> <span class="s1">&#39;Wilcoxon-</span><span class="si">{}</span><span class="s1">.csv&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">indicator_name</span><span class="p">)),</span> <span class="n">sep</span><span class="o">=</span><span class="s1">&#39;</span><span class="se">\t</span><span class="s1">&#39;</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="s1">&#39;utf-8&#39;</span><span class="p">)</span>

        <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">output_dir</span><span class="p">,</span> <span class="s1">&#39;Wilcoxon-</span><span class="si">{}</span><span class="s1">.tex&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">indicator_name</span><span class="p">)),</span> <span class="s1">&#39;w&#39;</span><span class="p">)</span> <span class="k">as</span> <span class="n">latex</span><span class="p">:</span>
            <span class="n">latex</span><span class="o">.</span><span class="n">write</span><span class="p">(</span>
                <span class="n">__wilcoxon_to_latex</span><span class="p">(</span>
                    <span class="n">table</span><span class="p">,</span>
                    <span class="n">caption</span><span class="o">=</span><span class="s1">&#39;Wilcoxon values of the </span><span class="si">{}</span><span class="s1"> quality indicator (</span><span class="si">{}</span><span class="s1">).&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">indicator_name</span><span class="p">,</span>
                                                                                       <span class="s1">&#39;, &#39;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">problems</span><span class="p">)),</span>
                    <span class="n">label</span><span class="o">=</span><span class="s1">&#39;table:</span><span class="si">{}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">indicator_name</span><span class="p">)</span>
                <span class="p">)</span>
            <span class="p">)</span>


<span class="k">def</span> <span class="nf">compute_mean_indicator</span><span class="p">(</span><span class="n">filename</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">indicator_name</span><span class="p">:</span> <span class="nb">str</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot; Compute the mean values of an indicator.</span>
<span class="sd">    :param filename:</span>
<span class="sd">    :param indicator_name: Quality indicator name.</span>
<span class="sd">    &quot;&quot;&quot;</span>
    <span class="n">df</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">read_csv</span><span class="p">(</span><span class="n">filename</span><span class="p">,</span> <span class="n">skipinitialspace</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>

    <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="nb">set</span><span class="p">(</span><span class="n">df</span><span class="o">.</span><span class="n">columns</span><span class="o">.</span><span class="n">tolist</span><span class="p">()))</span> <span class="o">!=</span> <span class="mi">5</span><span class="p">:</span>
        <span class="k">raise</span> <span class="ne">Exception</span><span class="p">(</span><span class="s1">&#39;Wrong number of columns&#39;</span><span class="p">)</span>

    <span class="n">algorithms</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">unique</span><span class="p">(</span><span class="n">df</span><span class="p">[</span><span class="s1">&#39;Algorithm&#39;</span><span class="p">])</span>
    <span class="n">problems</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">unique</span><span class="p">(</span><span class="n">df</span><span class="p">[</span><span class="s1">&#39;Problem&#39;</span><span class="p">])</span>

    <span class="c1"># We consider the quality indicator indicator_name</span>
    <span class="n">data</span> <span class="o">=</span> <span class="n">df</span><span class="p">[</span><span class="n">df</span><span class="p">[</span><span class="s1">&#39;IndicatorName&#39;</span><span class="p">]</span> <span class="o">==</span> <span class="n">indicator_name</span><span class="p">]</span>

    <span class="c1"># Compute for each pair algorithm/problem the average of IndicatorValue</span>
    <span class="n">average_values</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">((</span><span class="n">problems</span><span class="o">.</span><span class="n">size</span><span class="p">,</span> <span class="n">algorithms</span><span class="o">.</span><span class="n">size</span><span class="p">))</span>
    <span class="n">j</span> <span class="o">=</span> <span class="mi">0</span>
    <span class="k">for</span> <span class="n">alg</span> <span class="ow">in</span> <span class="n">algorithms</span><span class="p">:</span>
        <span class="n">i</span> <span class="o">=</span> <span class="mi">0</span>
        <span class="k">for</span> <span class="n">pr</span> <span class="ow">in</span> <span class="n">problems</span><span class="p">:</span>
            <span class="n">average_values</span><span class="p">[</span><span class="n">i</span><span class="p">,</span> <span class="n">j</span><span class="p">]</span> <span class="o">=</span> <span class="n">data</span><span class="p">[</span><span class="s1">&#39;IndicatorValue&#39;</span><span class="p">][</span><span class="n">np</span><span class="o">.</span><span class="n">logical_and</span><span class="p">(</span>
                <span class="n">data</span><span class="p">[</span><span class="s1">&#39;Algorithm&#39;</span><span class="p">]</span> <span class="o">==</span> <span class="n">alg</span><span class="p">,</span> <span class="n">data</span><span class="p">[</span><span class="s1">&#39;Problem&#39;</span><span class="p">]</span> <span class="o">==</span> <span class="n">pr</span><span class="p">)]</span><span class="o">.</span><span class="n">mean</span><span class="p">()</span>
            <span class="n">i</span> <span class="o">+=</span> <span class="mi">1</span>
        <span class="n">j</span> <span class="o">+=</span> <span class="mi">1</span>

    <span class="c1"># Generate dataFrame from average values and order columns by name</span>
    <span class="n">df</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">(</span><span class="n">data</span><span class="o">=</span><span class="n">average_values</span><span class="p">,</span> <span class="n">index</span><span class="o">=</span><span class="n">problems</span><span class="p">,</span> <span class="n">columns</span><span class="o">=</span><span class="n">algorithms</span><span class="p">)</span>
    <span class="n">df</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">reindex</span><span class="p">(</span><span class="n">df</span><span class="o">.</span><span class="n">columns</span><span class="p">,</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>

    <span class="k">return</span> <span class="n">df</span>


<span class="k">def</span> <span class="nf">__averages_to_latex</span><span class="p">(</span><span class="n">central_tendency</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">,</span> <span class="n">dispersion</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">,</span>
                        <span class="n">caption</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">label</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">minimization</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">alignment</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s1">&#39;c&#39;</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot; Convert a pandas DataFrame to a LaTeX tabular. Prints labels in bold and does use math mode.</span>

<span class="sd">    :param caption: LaTeX table caption.</span>
<span class="sd">    :param label: LaTeX table label.</span>
<span class="sd">    :param minimization: If indicator is minimization, highlight the best values of mean/median; else, the lowest.</span>
<span class="sd">    &quot;&quot;&quot;</span>
    <span class="n">num_columns</span><span class="p">,</span> <span class="n">num_rows</span> <span class="o">=</span> <span class="n">central_tendency</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">1</span><span class="p">],</span> <span class="n">central_tendency</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
    <span class="n">output</span> <span class="o">=</span> <span class="n">io</span><span class="o">.</span><span class="n">StringIO</span><span class="p">()</span>

    <span class="n">col_format</span> <span class="o">=</span> <span class="s1">&#39;</span><span class="si">{}</span><span class="s1">|</span><span class="si">{}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">alignment</span><span class="p">,</span> <span class="n">alignment</span> <span class="o">*</span> <span class="n">num_columns</span><span class="p">)</span>
    <span class="n">column_labels</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;</span><span class="se">\\</span><span class="s1">textbf{{</span><span class="si">{0}</span><span class="s1">}}&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">label</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39;_&#39;</span><span class="p">,</span> <span class="s1">&#39;</span><span class="se">\\</span><span class="s1">_&#39;</span><span class="p">))</span> <span class="k">for</span> <span class="n">label</span> <span class="ow">in</span> <span class="n">central_tendency</span><span class="o">.</span><span class="n">columns</span><span class="p">]</span>

    <span class="c1"># Write header</span>
    <span class="n">output</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="s1">&#39;</span><span class="se">\\</span><span class="s1">documentclass</span><span class="si">{article}</span><span class="se">\n</span><span class="s1">&#39;</span><span class="p">)</span>

    <span class="n">output</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="s1">&#39;</span><span class="se">\\</span><span class="s1">usepackage[utf8]</span><span class="si">{inputenc}</span><span class="se">\n</span><span class="s1">&#39;</span><span class="p">)</span>
    <span class="n">output</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="s1">&#39;</span><span class="se">\\</span><span class="s1">usepackage</span><span class="si">{tabularx}</span><span class="se">\n</span><span class="s1">&#39;</span><span class="p">)</span>
    <span class="n">output</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="s1">&#39;</span><span class="se">\\</span><span class="s1">usepackage</span><span class="si">{colortbl}</span><span class="se">\n</span><span class="s1">&#39;</span><span class="p">)</span>
    <span class="n">output</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="s1">&#39;</span><span class="se">\\</span><span class="s1">usepackage[table*]</span><span class="si">{xcolor}</span><span class="se">\n</span><span class="s1">&#39;</span><span class="p">)</span>

    <span class="n">output</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="s1">&#39;</span><span class="se">\\</span><span class="s1">xdefinecolor</span><span class="si">{gray95}{gray}{0.65}</span><span class="se">\n</span><span class="s1">&#39;</span><span class="p">)</span>
    <span class="n">output</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="s1">&#39;</span><span class="se">\\</span><span class="s1">xdefinecolor</span><span class="si">{gray25}{gray}{0.8}</span><span class="se">\n</span><span class="s1">&#39;</span><span class="p">)</span>

    <span class="n">output</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="s1">&#39;</span><span class="se">\\</span><span class="s1">title{Median and IQR}</span><span class="se">\n</span><span class="s1">&#39;</span><span class="p">)</span>
    <span class="n">output</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="s1">&#39;</span><span class="se">\\</span><span class="s1">author</span><span class="si">{}</span><span class="se">\n</span><span class="s1">&#39;</span><span class="p">)</span>

    <span class="n">output</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="s1">&#39;</span><span class="se">\\</span><span class="s1">begin</span><span class="si">{document}</span><span class="se">\n</span><span class="s1">&#39;</span><span class="p">)</span>
    <span class="n">output</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="s1">&#39;</span><span class="se">\\</span><span class="s1">maketitle</span><span class="se">\n</span><span class="s1">&#39;</span><span class="p">)</span>

    <span class="n">output</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="s1">&#39;</span><span class="se">\\</span><span class="s1">section</span><span class="si">{Table}</span><span class="se">\n</span><span class="s1">&#39;</span><span class="p">)</span>

    <span class="n">output</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="s1">&#39;</span><span class="se">\\</span><span class="s1">begin</span><span class="si">{table}</span><span class="s1">[!htp]</span><span class="se">\n</span><span class="s1">&#39;</span><span class="p">)</span>
    <span class="n">output</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="s1">&#39;  </span><span class="se">\\</span><span class="s1">caption{{</span><span class="si">{}</span><span class="s1">}}</span><span class="se">\n</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">caption</span><span class="p">))</span>
    <span class="n">output</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="s1">&#39;  </span><span class="se">\\</span><span class="s1">label{{</span><span class="si">{}</span><span class="s1">}}</span><span class="se">\n</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">label</span><span class="p">))</span>
    <span class="n">output</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="s1">&#39;  </span><span class="se">\\</span><span class="s1">centering</span><span class="se">\n</span><span class="s1">&#39;</span><span class="p">)</span>
    <span class="n">output</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="s1">&#39;  </span><span class="se">\\</span><span class="s1">begin</span><span class="si">{scriptsize}</span><span class="se">\n</span><span class="s1">&#39;</span><span class="p">)</span>
    <span class="n">output</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="s1">&#39;  </span><span class="se">\\</span><span class="s1">begin</span><span class="si">{tabular}</span><span class="s1">{</span><span class="si">%s</span><span class="s1">}</span><span class="se">\n</span><span class="s1">&#39;</span> <span class="o">%</span> <span class="n">col_format</span><span class="p">)</span>
    <span class="n">output</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="s1">&#39;      &amp; </span><span class="si">{}</span><span class="s1"> </span><span class="se">\\\\\\</span><span class="s1">hline</span><span class="se">\n</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="s1">&#39; &amp; &#39;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">column_labels</span><span class="p">)))</span>

    <span class="c1"># Write data lines</span>
    <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">num_rows</span><span class="p">):</span>
        <span class="n">central_values</span> <span class="o">=</span> <span class="p">[</span><span class="n">v</span> <span class="k">for</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">central_tendency</span><span class="o">.</span><span class="n">ix</span><span class="p">[</span><span class="n">i</span><span class="p">]]</span>
        <span class="n">dispersion_values</span> <span class="o">=</span> <span class="p">[</span><span class="n">v</span> <span class="k">for</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">dispersion</span><span class="o">.</span><span class="n">ix</span><span class="p">[</span><span class="n">i</span><span class="p">]]</span>

        <span class="c1"># Sort mean/median values (the lower the better if minimization)</span>
        <span class="c1"># Note that mean/median values could be the same: in that case, sort by Std/IQR (the lower the better)</span>
        <span class="n">sorted_values</span> <span class="o">=</span> <span class="nb">sorted</span><span class="p">(</span>
            <span class="nb">zip</span><span class="p">(</span><span class="n">central_values</span><span class="p">,</span> <span class="n">dispersion_values</span><span class="p">,</span> <span class="p">[</span><span class="n">i</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">central_values</span><span class="p">))]),</span> <span class="n">key</span><span class="o">=</span><span class="k">lambda</span> <span class="n">v</span><span class="p">:</span> <span class="p">(</span><span class="n">v</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="o">-</span><span class="n">v</span><span class="p">[</span><span class="mi">1</span><span class="p">])</span>
        <span class="p">)</span>

        <span class="k">if</span> <span class="n">minimization</span><span class="p">:</span>
            <span class="n">second_best</span><span class="p">,</span> <span class="n">best</span> <span class="o">=</span> <span class="n">sorted_values</span><span class="p">[</span><span class="mi">0</span><span class="p">][</span><span class="mi">2</span><span class="p">],</span> <span class="n">sorted_values</span><span class="p">[</span><span class="mi">1</span><span class="p">][</span><span class="mi">2</span><span class="p">]</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="n">second_best</span><span class="p">,</span> <span class="n">best</span> <span class="o">=</span> <span class="n">sorted_values</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">][</span><span class="mi">2</span><span class="p">],</span> <span class="n">sorted_values</span><span class="p">[</span><span class="o">-</span><span class="mi">2</span><span class="p">][</span><span class="mi">2</span><span class="p">]</span>

        <span class="c1"># Compose cell</span>
        <span class="n">values</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;</span><span class="si">{:.2e}</span><span class="s1">_{{</span><span class="si">{:.2e}</span><span class="s1">}}&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">central_values</span><span class="p">[</span><span class="n">i</span><span class="p">],</span> <span class="n">dispersion_values</span><span class="p">[</span><span class="n">i</span><span class="p">])</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span>
                  <span class="nb">range</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">central_values</span><span class="p">))]</span>

        <span class="c1"># Highlight values</span>
        <span class="n">values</span><span class="p">[</span><span class="n">best</span><span class="p">]</span> <span class="o">=</span> <span class="s1">&#39;</span><span class="se">\\</span><span class="s1">cellcolor</span><span class="si">{gray25}</span><span class="s1"> &#39;</span> <span class="o">+</span> <span class="n">values</span><span class="p">[</span><span class="n">best</span><span class="p">]</span>
        <span class="n">values</span><span class="p">[</span><span class="n">second_best</span><span class="p">]</span> <span class="o">=</span> <span class="s1">&#39;</span><span class="se">\\</span><span class="s1">cellcolor</span><span class="si">{gray95}</span><span class="s1"> &#39;</span> <span class="o">+</span> <span class="n">values</span><span class="p">[</span><span class="n">second_best</span><span class="p">]</span>

        <span class="n">output</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="s1">&#39;      </span><span class="se">\\</span><span class="s1">textbf{{</span><span class="si">{0}</span><span class="s1">}} &amp; $</span><span class="si">{1}</span><span class="s1">$ </span><span class="se">\\\\\n</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
            <span class="n">central_tendency</span><span class="o">.</span><span class="n">index</span><span class="p">[</span><span class="n">i</span><span class="p">],</span> <span class="s1">&#39; $ &amp; $ &#39;</span><span class="o">.</span><span class="n">join</span><span class="p">([</span><span class="nb">str</span><span class="p">(</span><span class="n">val</span><span class="p">)</span> <span class="k">for</span> <span class="n">val</span> <span class="ow">in</span> <span class="n">values</span><span class="p">]))</span>
        <span class="p">)</span>

    <span class="c1"># Write footer</span>
    <span class="n">output</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="s1">&#39;  </span><span class="se">\\</span><span class="s1">end</span><span class="si">{tabular}</span><span class="se">\n</span><span class="s1">&#39;</span><span class="p">)</span>
    <span class="n">output</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="s1">&#39;  </span><span class="se">\\</span><span class="s1">end</span><span class="si">{scriptsize}</span><span class="se">\n</span><span class="s1">&#39;</span><span class="p">)</span>
    <span class="n">output</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="s1">&#39;</span><span class="se">\\</span><span class="s1">end</span><span class="si">{table}</span><span class="se">\n</span><span class="s1">&#39;</span><span class="p">)</span>

    <span class="n">output</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="s1">&#39;</span><span class="se">\\</span><span class="s1">end</span><span class="si">{document}</span><span class="s1">&#39;</span><span class="p">)</span>

    <span class="k">return</span> <span class="n">output</span><span class="o">.</span><span class="n">getvalue</span><span class="p">()</span>


<span class="k">def</span> <span class="nf">__wilcoxon_to_latex</span><span class="p">(</span><span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">,</span> <span class="n">caption</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">label</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">minimization</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">alignment</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s1">&#39;c&#39;</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot; Convert a pandas DataFrame to a LaTeX tabular. Prints labels in bold and does use math mode.</span>

<span class="sd">    :param df: Pandas dataframe.</span>
<span class="sd">    :param caption: LaTeX table caption.</span>
<span class="sd">    :param label: LaTeX table label.</span>
<span class="sd">    :param minimization: If indicator is minimization, highlight the best values of mean/median; else, the lowest.</span>
<span class="sd">    &quot;&quot;&quot;</span>
    <span class="n">num_columns</span><span class="p">,</span> <span class="n">num_rows</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">1</span><span class="p">],</span> <span class="n">df</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
    <span class="n">output</span> <span class="o">=</span> <span class="n">io</span><span class="o">.</span><span class="n">StringIO</span><span class="p">()</span>

    <span class="n">col_format</span> <span class="o">=</span> <span class="s1">&#39;</span><span class="si">{}</span><span class="s1">|</span><span class="si">{}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">alignment</span><span class="p">,</span> <span class="n">alignment</span> <span class="o">*</span> <span class="n">num_columns</span><span class="p">)</span>
    <span class="n">column_labels</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;</span><span class="se">\\</span><span class="s1">textbf{{</span><span class="si">{0}</span><span class="s1">}}&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">label</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39;_&#39;</span><span class="p">,</span> <span class="s1">&#39;</span><span class="se">\\</span><span class="s1">_&#39;</span><span class="p">))</span> <span class="k">for</span> <span class="n">label</span> <span class="ow">in</span> <span class="n">df</span><span class="o">.</span><span class="n">columns</span><span class="p">]</span>

    <span class="c1"># Write header</span>
    <span class="n">output</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="s1">&#39;</span><span class="se">\\</span><span class="s1">documentclass</span><span class="si">{article}</span><span class="se">\n</span><span class="s1">&#39;</span><span class="p">)</span>

    <span class="n">output</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="s1">&#39;</span><span class="se">\\</span><span class="s1">usepackage[utf8]</span><span class="si">{inputenc}</span><span class="se">\n</span><span class="s1">&#39;</span><span class="p">)</span>
    <span class="n">output</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="s1">&#39;</span><span class="se">\\</span><span class="s1">usepackage</span><span class="si">{tabularx}</span><span class="se">\n</span><span class="s1">&#39;</span><span class="p">)</span>
    <span class="n">output</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="s1">&#39;</span><span class="se">\\</span><span class="s1">usepackage</span><span class="si">{amssymb}</span><span class="se">\n</span><span class="s1">&#39;</span><span class="p">)</span>
    <span class="n">output</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="s1">&#39;</span><span class="se">\\</span><span class="s1">usepackage</span><span class="si">{amsmath}</span><span class="se">\n</span><span class="s1">&#39;</span><span class="p">)</span>

    <span class="n">output</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="s1">&#39;</span><span class="se">\\</span><span class="s1">title{Wilcoxon - Mann-Whitney rank sum test}</span><span class="se">\n</span><span class="s1">&#39;</span><span class="p">)</span>
    <span class="n">output</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="s1">&#39;</span><span class="se">\\</span><span class="s1">author</span><span class="si">{}</span><span class="se">\n</span><span class="s1">&#39;</span><span class="p">)</span>

    <span class="n">output</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="s1">&#39;</span><span class="se">\\</span><span class="s1">begin</span><span class="si">{document}</span><span class="se">\n</span><span class="s1">&#39;</span><span class="p">)</span>
    <span class="n">output</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="s1">&#39;</span><span class="se">\\</span><span class="s1">maketitle</span><span class="se">\n</span><span class="s1">&#39;</span><span class="p">)</span>

    <span class="n">output</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="s1">&#39;</span><span class="se">\\</span><span class="s1">section</span><span class="si">{Table}</span><span class="se">\n</span><span class="s1">&#39;</span><span class="p">)</span>

    <span class="n">output</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="s1">&#39;</span><span class="se">\\</span><span class="s1">begin</span><span class="si">{table}</span><span class="s1">[!htp]</span><span class="se">\n</span><span class="s1">&#39;</span><span class="p">)</span>
    <span class="n">output</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="s1">&#39;  </span><span class="se">\\</span><span class="s1">caption{{</span><span class="si">{}</span><span class="s1">}}</span><span class="se">\n</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">caption</span><span class="p">))</span>
    <span class="n">output</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="s1">&#39;  </span><span class="se">\\</span><span class="s1">label{{</span><span class="si">{}</span><span class="s1">}}</span><span class="se">\n</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">label</span><span class="p">))</span>
    <span class="n">output</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="s1">&#39;  </span><span class="se">\\</span><span class="s1">centering</span><span class="se">\n</span><span class="s1">&#39;</span><span class="p">)</span>
    <span class="n">output</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="s1">&#39;  </span><span class="se">\\</span><span class="s1">begin</span><span class="si">{scriptsize}</span><span class="se">\n</span><span class="s1">&#39;</span><span class="p">)</span>
    <span class="n">output</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="s1">&#39;  </span><span class="se">\\</span><span class="s1">begin</span><span class="si">{tabular}</span><span class="s1">{</span><span class="si">%s</span><span class="s1">}</span><span class="se">\n</span><span class="s1">&#39;</span> <span class="o">%</span> <span class="n">col_format</span><span class="p">)</span>
    <span class="n">output</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="s1">&#39;      &amp; </span><span class="si">{}</span><span class="s1"> </span><span class="se">\\\\\\</span><span class="s1">hline</span><span class="se">\n</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="s1">&#39; &amp; &#39;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">column_labels</span><span class="p">)))</span>

    <span class="n">symbolo</span> <span class="o">=</span> <span class="s1">&#39;</span><span class="se">\\</span><span class="s1">triangledown\ &#39;</span>
    <span class="n">symbolplus</span> <span class="o">=</span> <span class="s1">&#39;</span><span class="se">\\</span><span class="s1">blacktriangle\ &#39;</span>

    <span class="k">if</span> <span class="ow">not</span> <span class="n">minimization</span><span class="p">:</span>
        <span class="n">symbolo</span><span class="p">,</span> <span class="n">symbolplus</span> <span class="o">=</span> <span class="n">symbolplus</span><span class="p">,</span> <span class="n">symbolo</span>

    <span class="c1"># Write data lines</span>
    <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">num_rows</span><span class="p">):</span>
        <span class="n">values</span> <span class="o">=</span> <span class="p">[</span><span class="n">val</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39;-&#39;</span><span class="p">,</span> <span class="s1">&#39;</span><span class="se">\\</span><span class="s1">text{--}\ &#39;</span><span class="p">)</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39;o&#39;</span><span class="p">,</span> <span class="n">symbolo</span><span class="p">)</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39;+&#39;</span><span class="p">,</span> <span class="n">symbolplus</span><span class="p">)</span> <span class="k">for</span> <span class="n">val</span> <span class="ow">in</span> <span class="n">df</span><span class="o">.</span><span class="n">ix</span><span class="p">[</span><span class="n">i</span><span class="p">]]</span>
        <span class="n">output</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="s1">&#39;      </span><span class="se">\\</span><span class="s1">textbf{{</span><span class="si">{0}</span><span class="s1">}} &amp; $</span><span class="si">{1}</span><span class="s1">$ </span><span class="se">\\\\\n</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
            <span class="n">df</span><span class="o">.</span><span class="n">index</span><span class="p">[</span><span class="n">i</span><span class="p">],</span> <span class="s1">&#39; $ &amp; $ &#39;</span><span class="o">.</span><span class="n">join</span><span class="p">([</span><span class="nb">str</span><span class="p">(</span><span class="n">val</span><span class="p">)</span> <span class="k">for</span> <span class="n">val</span> <span class="ow">in</span> <span class="n">values</span><span class="p">]))</span>
        <span class="p">)</span>

    <span class="c1"># Write footer</span>
    <span class="n">output</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="s1">&#39;  </span><span class="se">\\</span><span class="s1">end</span><span class="si">{tabular}</span><span class="se">\n</span><span class="s1">&#39;</span><span class="p">)</span>
    <span class="n">output</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="s1">&#39;  </span><span class="se">\\</span><span class="s1">end</span><span class="si">{scriptsize}</span><span class="se">\n</span><span class="s1">&#39;</span><span class="p">)</span>
    <span class="n">output</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="s1">&#39;</span><span class="se">\\</span><span class="s1">end</span><span class="si">{table}</span><span class="se">\n</span><span class="s1">&#39;</span><span class="p">)</span>

    <span class="n">output</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="s1">&#39;</span><span class="se">\\</span><span class="s1">end</span><span class="si">{document}</span><span class="s1">&#39;</span><span class="p">)</span>

    <span class="k">return</span> <span class="n">output</span><span class="o">.</span><span class="n">getvalue</span><span class="p">()</span>


<span class="k">def</span> <span class="nf">check_minimization</span><span class="p">(</span><span class="n">indicator</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">bool</span><span class="p">:</span>
    <span class="k">if</span> <span class="n">indicator</span> <span class="o">==</span> <span class="s1">&#39;HV&#39;</span><span class="p">:</span>
        <span class="k">return</span> <span class="kc">False</span>
    <span class="k">else</span><span class="p">:</span>
        <span class="k">return</span> <span class="kc">True</span>
</pre></div>

          </div>
            
        </div>
        <div class="clearfix"></div>
    </div>
    <div class="related" role="navigation" aria-label="related navigation">
      <h3>Navigation</h3>
      <ul>
        <li class="right" style="margin-right: 10px">
          <a href="../../../genindex.html" title="General Index"
             >index</a></li>
        <li class="right" >
          <a href="../../../py-modindex.html" title="Python Module Index"
             >modules</a> |</li>
        <li class="nav-item nav-item-0"><a href="../../../index.html">jMetalPy 1.5.3 documentation</a> &#187;</li>
          <li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> &#187;</li> 
      </ul>
    </div>
<script type="text/javascript">
  $("#mobile-toggle a").click(function () {
    $("#left-column").toggle();
  });
</script>
<script type="text/javascript" src="../../../_static/js/bootstrap.js"></script>
  <div class="footer">
    &copy; Copyright 2019, Antonio Benítez-Hidalgo. Created using <a href="http://sphinx.pocoo.org/">Sphinx</a>.
  </div>
  </body>
</html>